{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "龙曲良老师课程笔记/07_神经网络与全连接层.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyPUqeTlt2zz976oPMtdFIES",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jinxiqinghuan/learn_pytorch/blob/main/%E9%BE%99%E6%9B%B2%E8%89%AF%E8%80%81%E5%B8%88%E8%AF%BE%E7%A8%8B%E7%AC%94%E8%AE%B0/07_%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C%E4%B8%8E%E5%85%A8%E8%BF%9E%E6%8E%A5%E5%B1%82.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EpmesoejDE4T"
      },
      "source": [
        "# 神经网络与全连接层"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TQ1bhorSDMFS"
      },
      "source": [
        "import torch\r\n",
        "from torch.nn import functional as F\r\n",
        "from matplotlib import pyplot as plt\r\n",
        "import numpy as np"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I_cwgXXmBleO"
      },
      "source": [
        "## 交叉熵"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kgH4I_9lBlP6",
        "outputId": "eebfaa30-31cb-45e2-adf5-04b2c8596ebc"
      },
      "source": [
        "a = torch.full([4], 1/4)\r\n",
        "a"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([0.2500, 0.2500, 0.2500, 0.2500])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H8nCf5ME8N7V",
        "outputId": "2f63d965-d93e-4d1d-c1db-2a54ed81a7dd"
      },
      "source": [
        "a*torch.log2(a)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([-0.5000, -0.5000, -0.5000, -0.5000])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z_ivSunsDbSR",
        "outputId": "ac741d7e-449d-4cb1-b8a2-fe6be003e05a"
      },
      "source": [
        "-(a*torch.log2(a)).sum()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(2.)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U1J0jc8ZEN4K",
        "outputId": "6cd99e81-2dd3-469c-d89d-92c20dd8e433"
      },
      "source": [
        "a = torch.tensor([0.1, 0.1, 0.1, 0.7])\r\n",
        "a"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([0.1000, 0.1000, 0.1000, 0.7000])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Hnu1naZEEXzN",
        "outputId": "966cab53-4600-4e3b-ba30-b075f3d2f1b8"
      },
      "source": [
        "-(a*torch.log2(a)).sum()"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(1.3568)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sOu65TAREYRO",
        "outputId": "79e372d3-ef86-40f1-fd1c-313f79e9a1b1"
      },
      "source": [
        "a = torch.tensor([0.001, 0.001, 0.001, 0.999])\r\n",
        "-(a*torch.log2(a)).sum()"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(0.0313)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZbPygXAQMbK4"
      },
      "source": [
        "cross entropy"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pkFI9Ww-Eusx"
      },
      "source": [
        "x = torch.randn(1, 784)\r\n",
        "w = torch.randn(10, 784)"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6vW9-wWXMhEv"
      },
      "source": [
        "logits = x@w.t()"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6je2p06bNhiv",
        "outputId": "9329a463-a282-4748-9bc6-d5ee0f3a7144"
      },
      "source": [
        "logits"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[-26.5431,  -4.7924,  -6.0051,   0.9188,  10.6731,   1.8368,  51.4542,\n",
              "          25.9747, -66.1441,  52.7316]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q5_xOVQRMoIV"
      },
      "source": [
        "pred = F.softmax(logits, dim=1)"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2q3VjXDrNi_4",
        "outputId": "c5d543a8-2591-42d5-afce-826ab7429865"
      },
      "source": [
        "pred"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[2.9152e-35, 8.1444e-26, 2.4221e-26, 2.4616e-23, 4.2409e-19, 6.1645e-23,\n",
              "         2.1799e-01, 1.8745e-12, 0.0000e+00, 7.8201e-01]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DjvNav5kNl88",
        "outputId": "ea0ba6ef-5738-415c-dc0e-05bde60ad763"
      },
      "source": [
        "min(pred[0]), max(pred[0])"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor(0.), tensor(0.7820))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5Suh6WueNB-X"
      },
      "source": [
        "pred_log = torch.log(pred)"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KhEgssd2NLG_",
        "outputId": "7b02c811-b35a-4933-98e4-33a84d52e74b"
      },
      "source": [
        "F.cross_entropy(logits, torch.tensor([3])) # 已经包括了softmax和log的步骤"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(52.0586)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "obOVrWYCNRyF",
        "outputId": "88c401e4-6868-4c40-a807-f09583502745"
      },
      "source": [
        "F.nll_loss(pred_log, torch.tensor([3])) # 不实用cross_entropy， 自己算的方式"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(52.0586)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ui_2TZG9O4IP"
      },
      "source": [
        "## 多分类问题实战"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fNWVYZMMNWZK"
      },
      "source": [
        "w1, b1 = torch.randn(200, 784, requires_grad=True), torch.zeros(200, requires_grad=True)\r\n",
        "w2, b2 = torch.randn(200, 200, requires_grad=True), torch.zeros(200, requires_grad=True)\r\n",
        "w3, b3 = torch.randn(10, 200, requires_grad=True), torch.zeros(10, requires_grad=True)"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xF4AKNRbPSHV"
      },
      "source": [
        "def forward(x):\r\n",
        "  x = x@w1.t() + b1\r\n",
        "  x = F.relu(x)\r\n",
        "  x = x@w2.t() + b2\r\n",
        "  x = F.relu(x)\r\n",
        "  x = x@w3.t() + b3\r\n",
        "  x = F.relu(x)\r\n",
        "  return x"
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wFZ-I93sPnYk"
      },
      "source": [
        "learning_rate = 0.01\r\n",
        "epochs = 10\r\n",
        "optimizer = torch.optim.SGD([w1, b1, w2, b2, w3, b3], lr=learning_rate)"
      ],
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VV5VO3nIQgBj"
      },
      "source": [
        "criteon = torch.nn.CrossEntropyLoss()"
      ],
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 245
        },
        "id": "lUI5-2gXRDuP",
        "outputId": "89a51d64-b312-4d89-9e64-fb8e7ce1c8a2"
      },
      "source": [
        "for epoch in range(epochs):\r\n",
        "  for batch_idx, (data, target) in enumerate(train_loader):\r\n",
        "    data = data.view(-1, 28*28)\r\n",
        "    logits = forward(data)\r\n",
        "    loss = criteon(logits, target)\r\n",
        "    optimizer.zero_grad()\r\n",
        "    loss.backward()\r\n",
        "    optimizer.step()"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-41-86a42bc65391>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m   \u001b[0;32mfor\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_loader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m28\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m28\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mlogits\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcriteon\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogits\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'train_loader' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6DjIcoufSzE1"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}